{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {},
   "source": [
    "## Exploring the Banking dataset using Autolabel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff",
   "metadata": {},
   "source": [
    "#### Setup the API Keys for providers that you want to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "92993c83-4473-4e05-9510-f543b070c7d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: OPENAI_API_KEY=<key>\n"
     ]
    }
   ],
   "source": [
    "%env OPENAI_API_KEY=<key>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c246f85",
   "metadata": {},
   "source": [
    "#### Install the autolabel library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc181e31",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b014d1-f45c-4479-9acc-0d20870b1786",
   "metadata": {},
   "source": [
    "## Run the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from autolabel import LabelingAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c93fae0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"config_banking.json\") as f:\n",
    "     config = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "acb4a3de-fa84-4b94-b17a-7a6fac892a1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/f5/gb1rtdtn131956qb38lqdwc40000gn/T/ipykernel_411/3943655472.py:1: UserWarning: WARNING! logprobs is not default parameter.\n",
      "                logprobs was transferred to model_kwargs.\n",
      "                Please confirm that logprobs is what you intended.\n",
      "  agent = LabelingAgent(config=config)\n",
      "/var/folders/f5/gb1rtdtn131956qb38lqdwc40000gn/T/ipykernel_411/3943655472.py:1: UserWarning: WARNING! top_logprobs is not default parameter.\n",
      "                top_logprobs was transferred to model_kwargs.\n",
      "                Please confirm that top_logprobs is what you intended.\n",
      "  agent = LabelingAgent(config=config)\n"
     ]
    }
   ],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "92667a39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ded6757b4f54abb942975c07dd6dd9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0098 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 15      </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0007 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0098\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m15     \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0007\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">(</span><span style=\"color: #008000; text-decoration-color: #008000\">'You are an expert at understanding bank customers support complaints and queries.\\n\\nYou will answer with just </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">the the correct output label and nothing else.\\n\\nSome examples with their output answers are provided </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">below:\\n\\nInput: My transaction to pay for an item was returned to my account.\\nOutput: {\"label\": </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"reverted_card_payment?\"}\\n\\nInput: How do i get my salary in the account?\\nOutput: {\"label\": </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"receiving_money\"}\\n\\nInput: Can my daughter open an account?\\nOutput: {\"label\": \"age_limit\"}\\n\\nInput: Can I use </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">app to freeze account and dispute fraud?\\nOutput: {\"label\": \"compromised_card\"}\\n\\nInput: My disposable virtual </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">card will not work\\nOutput: {\"label\": \"virtual_card_not_working\"}\\n\\nInput: I tried to take out cash but the amount</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">isn\\'t right, so what do I do?\\nOutput: {\"label\": \"wrong_amount_of_cash_received\"}\\n\\nInput: What is going on? My </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">top-up is still pending. I use your system all the time but now it is just showing as pending.\\nOutput: {\"label\": </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"pending_top_up\"}\\n\\nInput: Can I unblock my pin?\\nOutput: {\"label\": \"pin_blocked\"}\\n\\nInput: I don\\'t recognize a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">debit payment that was made and would like to find out about the payment.\\nOutput: {\"label\": </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"direct_debit_payment_not_recognised\"}\\n\\nInput: I have withdrawn cash from ATM but i received the wrong amount. I </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">want cash back as in app its showing actual amount which i got. Please help me in this.\\nOutput: {\"label\": </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"wrong_amount_of_cash_received\"}\\n\\nNow I want you to label the following example:\\nInput: I want to close my </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">account\\nOutput: '</span>, <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'title'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'AnswerFormat'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'description'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Answer to the provided prompt.'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'object'</span>, \n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">'properties'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'label'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'title'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'label'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'string'</span><span style=\"font-weight: bold\">}}</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'required'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'label'</span><span style=\"font-weight: bold\">]</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'additionalProperties'</span>: \n",
       "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'definitions'</span>: <span style=\"font-weight: bold\">{}})</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m(\u001b[0m\u001b[32m'You are an expert at understanding bank customers support complaints and queries.\\n\\nYou will answer with just \u001b[0m\n",
       "\u001b[32mthe the correct output label and nothing else.\\n\\nSome examples with their output answers are provided \u001b[0m\n",
       "\u001b[32mbelow:\\n\\nInput: My transaction to pay for an item was returned to my account.\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \u001b[0m\n",
       "\u001b[32m\"reverted_card_payment?\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: How do i get my salary in the account?\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \u001b[0m\n",
       "\u001b[32m\"receiving_money\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: Can my daughter open an account?\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \"age_limit\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: Can I use \u001b[0m\n",
       "\u001b[32mapp to freeze account and dispute fraud?\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \"compromised_card\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: My disposable virtual \u001b[0m\n",
       "\u001b[32mcard will not work\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \"virtual_card_not_working\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: I tried to take out cash but the amount\u001b[0m\n",
       "\u001b[32misn\\'t right, so what do I do?\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \"wrong_amount_of_cash_received\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: What is going on? My \u001b[0m\n",
       "\u001b[32mtop-up is still pending. I use your system all the time but now it is just showing as pending.\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \u001b[0m\n",
       "\u001b[32m\"pending_top_up\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: Can I unblock my pin?\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \"pin_blocked\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: I don\\'t recognize a \u001b[0m\n",
       "\u001b[32mdebit payment that was made and would like to find out about the payment.\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \u001b[0m\n",
       "\u001b[32m\"direct_debit_payment_not_recognised\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nInput: I have withdrawn cash from ATM but i received the wrong amount. I \u001b[0m\n",
       "\u001b[32mwant cash back as in app its showing actual amount which i got. Please help me in this.\\nOutput: \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"label\": \u001b[0m\n",
       "\u001b[32m\"wrong_amount_of_cash_received\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\n\\nNow I want you to label the following example:\\nInput: I want to close my \u001b[0m\n",
       "\u001b[32maccount\\nOutput: '\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'title'\u001b[0m: \u001b[32m'AnswerFormat'\u001b[0m, \u001b[32m'description'\u001b[0m: \u001b[32m'Answer to the provided prompt.'\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'object'\u001b[0m, \n",
       "\u001b[32m'properties'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'label'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'title'\u001b[0m: \u001b[32m'label'\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'string'\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m, \u001b[32m'required'\u001b[0m: \u001b[1m[\u001b[0m\u001b[32m'label'\u001b[0m\u001b[1m]\u001b[0m, \u001b[32m'additionalProperties'\u001b[0m: \n",
       "\u001b[3;91mFalse\u001b[0m, \u001b[32m'definitions'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c435dca05324e90a9c05506fa41c6b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Actual Cost: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Actual Cost: \u001b[1;36m0.0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                      example              label  \\\n",
      "0  I want to close my account  terminate_account   \n",
      "\n",
      "  BankingComplaintsClassification_task_label      label_label  \\\n",
      "0               {'label': 'closing_account'}  closing_account   \n",
      "\n",
      "  BankingComplaintsClassification_task_error  \\\n",
      "0                                       None   \n",
      "\n",
      "         BankingComplaintsClassification_task_prompt  \\\n",
      "0  You are an expert at understanding bank custom...   \n",
      "\n",
      "   BankingComplaintsClassification_task_successfully_labeled  \\\n",
      "0                                               True           \n",
      "\n",
      "     BankingComplaintsClassification_task_annotation  \\\n",
      "0  successfully_labeled=True label={'label': 'clo...   \n",
      "\n",
      "  BankingComplaintsClassification_task_confidence  label_confidence  \n",
      "0                   {'label': 0.8701322819008598}          0.870132  \n"
     ]
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=1)\n",
    "print(ds.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31250a7f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
